﻿using JiebaNet.Segmenter;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace ConsoleApp1;

public class ChineseTextSimilarity
{
    private readonly JiebaSegmenter _segmenter;
    private readonly HashSet<string> _stopWords;

    public ChineseTextSimilarity(string stopWordsFilePath = "stopwords.txt")
    {
        _segmenter = new JiebaSegmenter();
        _segmenter.LoadUserDict("userdict.txt");
        _stopWords = LoadStopWords(stopWordsFilePath);
    }

    private HashSet<string> LoadStopWords(string filePath)
    {
        return File.ReadAllLines(filePath)
            .Select(line => line.Trim())
            .Where(line => !string.IsNullOrEmpty(line))
            .ToHashSet();
    }

    private List<string> SegmentAndClean(string text)
    {
        var ret = _segmenter.Cut(text, cutAll: true)
            .Select(word => word.ToLowerInvariant())
            .Where(word => !_stopWords.Contains(word) && word.Length > 1)
            .ToList();

        return ret;
    }

    // 使用Jaccard相似度
    public double CalculateJaccardSimilarity(string text1, string text2)
    {
        var set1 = SegmentAndClean(text1).ToHashSet();
        var set2 = SegmentAndClean(text2).ToHashSet();

        var intersection = set1.Intersect(set2).Count();
        var union = set1.Union(set2).Count();

        return union == 0 ? 0 : (double)intersection / union;
    }

    // 使用余弦相似度（更精确）
    public double CalculateCosineSimilarity(string text1, string text2)
    {
        var tokens1 = SegmentAndClean(text1);
        var tokens2 = SegmentAndClean(text2);

        var allWords = tokens1.Union(tokens2).Distinct().ToList();

        var vector1 = allWords.Select(word => tokens1.Count(w => w == word)).ToArray();
        var vector2 = allWords.Select(word => tokens2.Count(w => w == word)).ToArray();

        var dotProduct = vector1.Zip(vector2, (a, b) => a * b).Sum();
        var magnitude1 = Math.Sqrt(vector1.Sum(x => x * x));
        var magnitude2 = Math.Sqrt(vector2.Sum(x => x * x));

        return magnitude1 * magnitude2 == 0
            ? 0
            : dotProduct / (magnitude1 * magnitude2);
    }
}
